using System;
using System.Collections.Generic;
using System.Text;
using System.IO;
using System.Collections;
using System.Collections.Specialized;
using System.Threading;
using System.Web.Services.Protocols;
using System.Xml.Serialization;
using System.Xml;
using MapPointBatchGeocoder;
using MapPointBatchGeocoder.MapPointCustomerDataServices;

namespace Geospatial
{
    /// <summary>
    /// This calls encapsulates the logic necessary to geocode a batch of address using MapPoint's Customer Data Services.  The class is designed to be 
    /// independent of SSIS so that users could leverage this code for other, non-SSIS purposes.
    /// </summary>
    public class Geocoder : IDisposable 
    {
        #region Constants
        private const int BATCH_INITIAL_CAPACITY = 1000000;         // initial size of the buffer used to store address data
        private const int UPLOAD_CHUNK_SIZE = 50000;                // when uploading files to MapPoint, used to break down the file into chunks for multiple calls
        private const int JOB_STATUS_POLL_DELAY_STAGING = 5;        // delay when polling the job status in staging
        private const int JOB_STATUS_POLL_DELAY_PRODUCTION = 20;    // delay when polling the job status in production
        private const string FILE_HEADER = "EntityID,AddressLine,PrimaryCity,Subdivision,PostalCode,CountryRegion,Phone,Latitude,Longitude";        // column headers for both the uploaded and downloaded files
        private const int CURSOR_UNINITIALIZED = -1;                // this value indicates the cursor, used when reading geocoded data, is uninitialized
        private const int CURSOR_EOF = -2;                          // this value indicates the cursor, used when reading geocoded data, has reached the end
        private const int HEADER_OFFSET = 2;                        // we must skip 2 rows when reading the geocoded data (an XML declaration and the header)
        private const int INDEX_HEADER_ENTITYID = 0;                // these indexes reference the associated field in the downloaded geocoded file
        private const int INDEX_HEADER_ADDRESSLINE = 1;
        private const int INDEX_HEADER_PRIMARYCITY = 2;
        private const int INDEX_HEADER_SUBDIVISION = 3;
        private const int INDEX_HEADER_POSTALCODE = 4;
        private const int INDEX_HEADER_COUNTRYREGION = 5;
        private const int INDEX_HEADER_PHONE = 6;
        private const int INDEX_HEADER_LATITUDE = 7;
        private const int INDEX_HEADER_LONGITUDE = 8;
        private const int INDEX_HEADER_MATCHCODE = 9;
        #endregion

        #region Private Member Variables
        private CDSRetryProxy cdsRetry;                                 // MapPoint Customer Data Services web proxy
        private System.Net.NetworkCredential credentials;               // MapPoint credentials, used in multiple web service calls
        private MemoryStream dataStream;                                // holds the pre-geocoded data from calling AddAddress
        private bool useStaging = true;                                 // true if we should use the staging environment
        private string dataSourceName = "";                             // the MapPoint data source we should use
        private string entityTypeName = "";                             // the MapPoint entity type name we should use
        private bool useStubCode;                                       // if true, indicates we should fabricate the lat/lon values and avoid a web service call
        private int addressCount;                                       // set to the number of addresses the caller has added by invoking AddAddress
        private string[] charLineSeparator = new string[] { "\r\n" };   // CRLF used in split operations
        private char[] charCommaSeparator = new char[] { ',' };         // comma used in split operations
        private string[] geocodedRows;                                  // an array of the geocoded data for read operations
        private string[] geocodedFields;                                // an array of geocoded fields from the current row for read operations
        private int cursorPos = CURSOR_UNINITIALIZED;                   // the current position of the read cursor (rows)
        private bool isGeocoded;                                        // set to true once the current batch is geocoded
        #endregion

        // This event is thrown externally whenever a meaningful operation
        // has taken place.  This allows the caller to consume this event
        // and log appropriately according to their configuration.
        public EventHandler<LogVerboseEventArgs> RaiseLogVerboseEvent;

        /// <summary>
        /// Initializes a new Geocoder component to perform batch geocoding. 
        /// </summary>
        /// <param name="mappointUserName">The username of the MapPoint account.</param>
        /// <param name="mappointPassword">The password of the MapPoint account.</param>
        /// <param name="dataSourceName">The name of the data source to use for geocoding (i.e. MyAccount123456.123456.Customer). The datasource will be created if it does not exist.</param>
        /// <param name="entityTypeName">The name of the entity type to overwrite</param>
        /// <param name="useStaging">Set to True to use the MapPoint staging environment.</param>
        /// <param name="retryAttempts">Number of attempts to retry accessing MapPoint services in case of network failure.</param>
        /// <param name="retryDelay">The delay between retries (in seconds).</param>
        /// <param name="stubData">If true, will cause the transformation to avoid any Web Service calls and instead will synthesis fake lat/lon values instead of invoking the service. Useful for certain testing scenarios.</param>
        public Geocoder(string mappointUserName, string mappointPassword, string dataSourceName, string entityTypeName, bool useStaging, int retryAttempts, int retryDelay, bool stubData)
        {
            this.dataSourceName = dataSourceName;
            this.entityTypeName = entityTypeName;
            this.useStaging = useStaging;
            this.useStubCode = stubData;

            this.cdsRetry = new CDSRetryProxy(retryAttempts, retryDelay);
            this.credentials = new System.Net.NetworkCredential(mappointUserName, mappointPassword);
            this.cdsRetry.Credentials = this.credentials;
            this.cdsRetry.PreAuthenticate = true;

            // Call our internal Reset() to prepare for usage
            Reset();
        }

        /// <summary>
        /// Perform cleanup of unmanaged resources
        /// </summary>
        #region IDisposable implementation
        protected virtual void Dispose(bool disposing)
        {
            if (disposing)
            {
                // Clean-up MapPoint data source before
                if (!useStubCode) { this.DeleteDataSource(); }

                // Clean up the data stream
                if (this.dataStream != null)
                {
                    this.dataStream.Dispose();
                    this.dataStream = null;
                }

                // Clean up our CDS web service proxy
                if (this.cdsRetry != null)
                {
                    this.cdsRetry.Dispose();
                    this.cdsRetry = null;
                }
                
            }
        }

        public void Dispose()
        {
            Dispose(true);
            GC.SuppressFinalize(this);
        }
        #endregion

        /// <summary>
        /// Adds an Address to be geocoded.  Once the entire list of addresses have been added, call ExecuteBatch to geocode.
        /// <remarks>This overloaded method adds only the minimum fields for MapPoint geocoding.</remarks>
        /// </summary>
        /// <param name="entityID">The unique identifier for this address</param>
        /// <param name="addressLine">The address line of the address</param>
        /// <param name="primaryCity">The city of the address</param>
        /// <param name="postalCode">The postalcode of the address</param>
        public void AddAddress(string entityID, string addressLine, string primaryCity, string postalCode)
        {
            AppendTextToStream(entityID + "," + addressLine + "," + primaryCity + ",," + postalCode + ",,,,");
            addressCount++;
        }

        /// <summary>
        /// Adds an Address to be geocoded.  Once the entire list of addresses have been added, call ExecuteBatch to geocode.
        /// <remarks>This overloaded method adds the full set of supported fields for MapPoint geocoding.</remarks>
        /// </summary>
        /// <param name="entityID">The unique identifier for this address</param>
        /// <param name="addressLine">The address line of the address</param>
        /// <param name="primaryCity">The city of the address</param>
        /// <param name="subdivision"></param>
        /// <param name="postalCode">The postalcode of the address</param>
        /// <param name="countryRegion"></param>
        /// <param name="phone"></param>
        public void AddAddress(string entityID, string addressLine, string primaryCity, string subdivision, string postalCode, string countryRegion, string phone)
        {
            AppendTextToStream(entityID + "," + addressLine + "," + primaryCity + "," + subdivision + "," + postalCode + "," + countryRegion + "," + phone + ",,");
            addressCount++;
        }

        /// <summary>
        /// Geocodes the current batch by orchestrating multiple steps using MapPoint Customer Data Services.  After this method is called, the geocoded values
        /// can be obtained by calling Read() and Get[FIELD].
        /// </summary>
        /// <param name="dataStream">Data to geocode</param>
        /// <param name="dataSourceName"></param>
        /// <param name="entityTypeName"></param>
        /// <returns></returns>
        public void ExecuteBatch()
        {
            if (addressCount == 0) { throw new ApplicationException("Cannot call ExecuteBatch when there are no addresses to geocode."); }

            // Remove any excess bytes considering we pre-allocated.
            dataStream.Capacity = (int)dataStream.Length;
            string downloadedFile = "";

            // Determine if we should stub the geocoding, or invoke the actual service
            if (!useStubCode)
            {
                downloadedFile = ExecuteBatchMapPoint();
            }
            else
            {
                downloadedFile = ExecuteBatchStubbed();
            }

            // At this point, we have a byte representation of the geocoded elements.  We need
            // to convert to a string array of each row in preparation for reading.
            geocodedRows = downloadedFile.Split(charLineSeparator, StringSplitOptions.RemoveEmptyEntries);
            downloadedFile = "";

            // Initialize some additional variables for reading.
            cursorPos = CURSOR_UNINITIALIZED;
            isGeocoded = true;
        }

        /// <summary>
        /// Geocodes the current batch using the MapPoint service.
        /// </summary>
        /// <returns>The geocoded file downloaded from MapPoint.
        /// <remarks>See the MapPoint documentation for details on the file format.</remarks>
        /// </returns>
        private string ExecuteBatchMapPoint()
        {
            // Create the upload specification
            UploadSpecification specificationUL = new UploadSpecification();
            specificationUL.Environment = (useStaging ? LocationDataEnvironment.Staging : LocationDataEnvironment.Service);
            specificationUL.EntityTypeName = entityTypeName;
            specificationUL.GeometryType = GeometryType.Point;
            specificationUL.MaximumGeocodingLevel = GeocodingLevel.Street;
            specificationUL.RejectAmbiguousGeocodes = false;
            specificationUL.DataSourceName = dataSourceName;

            // Start Batch Geocode
            string jobID = cdsRetry.StartUpload(specificationUL);

            // Upload data
            long filesize = dataStream.Length;
            long bytesPreviouslyUploaded = 0;
            dataStream.Position = 0;
            while (bytesPreviouslyUploaded < filesize)
            {
                long remainingBytes = filesize - bytesPreviouslyUploaded;
                byte[] localBuffer = new byte[UPLOAD_CHUNK_SIZE < remainingBytes ? UPLOAD_CHUNK_SIZE : remainingBytes];
                dataStream.Read(localBuffer, 0, localBuffer.Length);

                bytesPreviouslyUploaded = cdsRetry.UploadData(jobID, localBuffer, bytesPreviouslyUploaded);

                OnLogVerbose("Uploaded " + bytesPreviouslyUploaded.ToString() + " out of " + filesize.ToString() + " bytes");
            }

            // Call FinishUpload method to kick-off geo-coding
            // processing the file on the MapPoint server
            cdsRetry.FinishUpload(jobID, filesize);

            // Monitor processing by polling the server for status
            WaitForJobToFinish(jobID);

            // Now that the upload has finished and processed, the file should be ready for downloading.

            // Create the download specification
            DownloadSpecification specification = new DownloadSpecification();
            specification.Environment = (useStaging ? LocationDataEnvironment.Staging : LocationDataEnvironment.Service);
            specification.Compressed = false;
            specification.DataSourceName = dataSourceName;
            specification.EntityTypeName = entityTypeName;
            specification.Format = FileFormat.CommaDelimitedTextLatin1;

            // Begin the download
            jobID = cdsRetry.StartDownload(specification);
            OnLogVerbose("JobID = \"" + jobID + "\"");

            // Monitor processing by polling the server 
            // for status
            WaitForJobToFinish(jobID);

            // Download the file
            string downloadURL = cdsRetry.GetDownloadFileURL(jobID);
            System.Net.WebClient webClient = new System.Net.WebClient();
            webClient.Credentials = this.credentials;

            string content = webClient.DownloadString(downloadURL);

            return content;
        }

        /// <summary>
        /// Fakes the geocoding of the current batch using random numbers for lat/lon.
        /// </summary>
        /// <returns></returns>
        private string ExecuteBatchStubbed()
        {
            OnLogVerbose("Generating stubbed geocoding data (MapPoint Services will not be called.  Please check the 'Service Stub MapPoint Data' option for more info).");

            // Convert the stream into a text block
            byte[] localBuffer = new byte[dataStream.Length];
            dataStream.Position = 0;
            dataStream.Read(localBuffer, 0, (int)dataStream.Length);
            string data = System.Text.Encoding.Default.GetString(localBuffer);

            // Prepare the variables used to randomize the geocoding values
            Random rnd = new Random();
            StringBuilder outputFile = new StringBuilder(data.Length);
            string[] sampleMatchCodes = { "Good", "Good", "Good", "Good", "Bad", "Imprecise", "Ambiguous" };      // give 4 to 1 of Good to other values to make more realistic

            // Break the batch down into rows
            string[] rows = data.Split(new string[] { "\r\n" }, StringSplitOptions.RemoveEmptyEntries);

            // Prepare the output buffer by adding the header
            outputFile.Append("<encoding name='UTF-8'>\r\n");     // append XML header declaration
            outputFile.Append(rows[0] + "\r\n");     // then append header

            // For each row in the batch
            for (int iRow = 1; iRow < rows.Length; iRow++ )     // skip the header
            {
                string[] cols = rows[iRow].Split(new string[] { "," }, StringSplitOptions.None);

                // Add the original values back to the output
                outputFile.Append(cols[0] + ",");
                outputFile.Append(cols[1] + ",");
                outputFile.Append(cols[2] + ",");
                outputFile.Append(cols[3] + ",");
                outputFile.Append(cols[4] + ",");
                outputFile.Append(cols[5] + ",");
                outputFile.Append(cols[6] + ",");

                // Now add the randomized geocoded values (including match code)
                outputFile.Append((rnd.NextDouble() * 90).ToString() + ",");
                outputFile.Append((rnd.NextDouble() * 180).ToString() + ",");       // ignores possibility of negative longitude, but good enough for testing
                outputFile.Append(sampleMatchCodes[rnd.Next(0, 6)] + "\r\n");
            }

            return outputFile.ToString();
        }

        /// <summary>
        /// Reads the next row of geocoded data.  Use the Get[FIELD] properties to retrieve the geocoded columns of data.
        /// <remarks>This method can only be called after ExecuteBatch().</remarks>
        /// </summary>
        /// <returns>True if another row is available, or false if the EOF is reached.</returns>
        public bool Read()
        {
            // Verify we have another row to read
            if ((cursorPos + HEADER_OFFSET + 1) >= geocodedRows.Length || cursorPos == CURSOR_EOF)
            {
                // We do not, so close the cursor
                cursorPos = CURSOR_EOF;
                return false;
            }
            else if (!isGeocoded)
            {
                throw new ApplicationException("Cannot call Read before ExecuteBatch has been invoked.");
            }

            cursorPos++;        // advance the cursor

            // Read the fields of the current row
            geocodedFields = geocodedRows[cursorPos + HEADER_OFFSET].Split(charCommaSeparator);

            return true;
        }

        /// <summary>
        /// Prepares the Geocoder for another batch by resetting the component's state.
        /// </summary>
        public void Reset()
        {
            this.isGeocoded = false;
            this.cursorPos = CURSOR_UNINITIALIZED;
            this.geocodedRows = null;
            this.geocodedFields = null;
            this.addressCount = 0;
            this.dataStream = null;
            this.dataStream = new MemoryStream(BATCH_INITIAL_CAPACITY);
            
            AppendTextToStream(FILE_HEADER);     // add the file header
        }

        /// <summary>The size of the current batch stream in bytes.</summary>
        public long BatchSize
        { get { return dataStream.Length; } }

        /// <summary>The number of addresses added to to the batch.</summary>
        public int AddressCount
        { get { return addressCount; } }

        /// <summary>The PostalCode value returned from MapPoint.</summary>
        public string GetPostalCode
        { get { return geocodedFields[INDEX_HEADER_POSTALCODE]; } }

        /// <summary>The AddressLine value returned from MapPoint.</summary>
        public string GetAddressLine
        { get { return geocodedFields[INDEX_HEADER_ADDRESSLINE]; } }

        /// <summary>The CountryRegion value returned from MapPoint.</summary>
        public string GetCountryRegion
        { get { return geocodedFields[INDEX_HEADER_COUNTRYREGION]; } }

        /// <summary>The EntityID value returned from MapPoint.</summary>
        public string GetEntityID
        { get { return geocodedFields[INDEX_HEADER_ENTITYID]; } }

        /// <summary>The Latitude value returned from MapPoint.</summary>
        public string GetLatitude
        { get { return geocodedFields[INDEX_HEADER_LATITUDE]; } }

        /// <summary>The Longitude value returned from MapPoint.</summary>
        public string GetLongitude
        { get { return geocodedFields[INDEX_HEADER_LONGITUDE]; } }

        /// <summary>The MatchCode value returned from MapPoint.</summary>
        public string GetMatchCode
        { get { return geocodedFields[INDEX_HEADER_MATCHCODE]; } }

        /// <summary>The Phone value returned from MapPoint.</summary>
        public string GetPhone
        { get { return geocodedFields[INDEX_HEADER_PHONE]; } }

        /// <summary>The PrimaryCity value returned from MapPoint.</summary>
        public string GetPrimaryCity
        { get { return geocodedFields[INDEX_HEADER_PRIMARYCITY]; } }

        /// <summary>The Subdivision value returned from MapPoint.</summary>
        public string GetSubdivision
        { get { return geocodedFields[INDEX_HEADER_SUBDIVISION]; } }

        /// <summary>
        /// Adds the specified text to the batch stream.
        /// </summary>
        /// <param name="text">Text to add to the batch stream</param>
        private void AppendTextToStream(string text)
        {
            byte[] textBytes = Encoding.ASCII.GetBytes(text + "\r\n");
            dataStream.Write(textBytes, 0, textBytes.Length);
        }

        /// <summary>
        /// Returns the delay (in seconds) used when polling job status from MapPoint services.  This value is different between production and staging, 
        /// production being higher, since production tends to geocode larger amounts of data and can poll less frequently.
        /// </summary>
        private int JobStatusPollDelay
        {
            get { return (useStaging ? JOB_STATUS_POLL_DELAY_STAGING : JOB_STATUS_POLL_DELAY_PRODUCTION); }
        }

        /// <summary>
        /// Deletes the current MapPoint data source.
        /// </summary>
        private void DeleteDataSource()
        {
            OnLogVerbose("Deleting data source '" + dataSourceName + "'.");

            LocationDataEnvironment ld = (useStaging ? LocationDataEnvironment.Staging : LocationDataEnvironment.Service);

            string JobID = cdsRetry.DeleteDataSource(dataSourceName, ld);
            WaitForJobToFinish(JobID);
            return;

        }
        
        /// <summary>
        /// Causes the current thread to sleep (JobStatusPollDelay) until a given MapPoint batch job has completed.
        /// </summary>
        /// <param name="jobID">ID of the MapPoint batch job</param>
        private void WaitForJobToFinish(string jobID)
        {
            JobState jobState = cdsRetry.GetJobState(jobID);

            while (IsWaitingState(jobState))
            {
                OnLogVerbose("Waiting: JobState is \"" + jobState.ToString() + "\"");

                Thread.Sleep(JobStatusPollDelay * 1000);
                jobState = cdsRetry.GetJobState(jobID);
            }

            // Verify the status isn't Loading, since at this point would mean FinishUpload was missed.
            if (jobState == JobState.Loading) { throw new ApplicationException("JobState is still \"Loading\".  FinishUpload was never called."); }

            OnLogVerbose("Final JobState is \"" + jobState.ToString() + "\"");
        }

        /// <summary>
        /// Checks if the given job is still processing.
        /// </summary>
        static private bool IsWaitingState(JobState state)
        {
            switch (state)
            {
                case JobState.Pending:
                case JobState.InProcess:
                    return true;
                default:
                    return false;
            }
        }

        /// <summary>
        /// Called internally when any useful event is encountered. Will throw a LogVerbose event to the caller for logging purposes.
        /// </summary>
        /// <param name="message">A description of the event.</param>
        private void OnLogVerbose(string message)
        {
            // Event will be null if there are no subscribers
            if (RaiseLogVerboseEvent != null)
            {
                LogVerboseEventArgs args = new LogVerboseEventArgs(message);
                RaiseLogVerboseEvent(this, args);
            }
        }
    }

    public class LogVerboseEventArgs : EventArgs
    {
        private string msg;
        public LogVerboseEventArgs(string message) { msg = message; }
        public string Message { get { return msg; } }
    }
}